{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/I348655/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import run_classifier as rc\n",
    "import tokenization\n",
    "import modeling\n",
    "import tensorflow as tf\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_dir=\"./DATA_DIR/test.tsv\"\n",
    "train_dir=\"./DATA_DIR/train.tsv\"\n",
    "init_checkpoint=\"./BERT_BASE_DIR/bert_model.ckpt\"\n",
    "bert_config_file=\"./BERT_BASE_DIR/bert_config.json\"\n",
    "output_dir='./output_models/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_examples = None\n",
    "num_train_steps = None\n",
    "num_warmup_steps = None\n",
    "learning_rate=2e-5\n",
    "use_tpu=False\n",
    "tpu_cluster_resolver = None\n",
    "master=None\n",
    "save_checkpoints_steps=50\n",
    "iterations_per_loop=50\n",
    "num_tpu_cores=8\n",
    "is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2\n",
    "train_batch_size=32\n",
    "eval_batch_size=8\n",
    "predict_batch_size=8\n",
    "predict_drop_remainder=False\n",
    "max_seq_length=128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class InputExample(object):\n",
    "    \"\"\"A single training/test example for simple sequence classification.\"\"\"\n",
    "\n",
    "    def __init__(self, guid, text_a, text_b=None, label=None):\n",
    "        \"\"\"Constructs a InputExample.\n",
    "\n",
    "        Args:\n",
    "          guid: Unique id for the example.\n",
    "          text_a: string. The untokenized text of the first sequence. For single\n",
    "            sequence tasks, only this sequence must be specified.\n",
    "          text_b: (Optional) string. The untokenized text of the second sequence.\n",
    "            Only must be specified for sequence pair tasks.\n",
    "          label: (Optional) string. The label of the example. This should be\n",
    "            specified for train and dev examples, but not for test examples.\n",
    "        \"\"\"\n",
    "        self.guid = guid\n",
    "        self.text_a = text_a\n",
    "        self.text_b = text_b\n",
    "        self.label = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_tsv(nput_file, quotechar=None):\n",
    "    \"\"\"Reads a tab separated value file.\"\"\"\n",
    "    with tf.gfile.Open(input_file, \"r\") as f:\n",
    "        reader = csv.reader(f, delimiter=\"\\t\", quotechar=quotechar)\n",
    "        lines = []\n",
    "        for line in reader:\n",
    "            lines.append(line)\n",
    "        return lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_examples(lines,set_type):\n",
    "    \"\"\"Creates examples for the training and dev sets.\"\"\"\n",
    "    examples = []\n",
    "    labels = []\n",
    "    labels_test = []\n",
    "    for (i, line) in enumerate(lines):\n",
    "        if i == 0:\n",
    "            continue\n",
    "        guid = \"%s-%s\" % (\"test\", i)\n",
    "\n",
    "        # tokenization is based on vocab file\n",
    "        text_a = tokenization.convert_to_unicode(line[0])\n",
    "        if set_type == \"test\":\n",
    "            label = \"0\"\n",
    "        else:\n",
    "            label = tokenization.convert_to_unicode(line[1])\n",
    "            labels.append(label)\n",
    "        labels_test.append(label)\n",
    "        examples.append(\n",
    "            InputExample(guid=guid, text_a=text_a, text_b=None, label=label))\n",
    "\n",
    "    return examples, labels, labels_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate,\n",
    "                     num_train_steps, num_warmup_steps, use_tpu,\n",
    "                     use_one_hot_embeddings):\n",
    "    \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n",
    "\n",
    "    def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument\n",
    "        \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n",
    "\n",
    "        tf.logging.info(\"*** Features ***\")\n",
    "        for name in sorted(features.keys()):\n",
    "            tf.logging.info(\"  name = %s, shape = %s\" % (name, features[name].shape))\n",
    "\n",
    "        input_ids = features[\"input_ids\"]\n",
    "        input_mask = features[\"input_mask\"]\n",
    "        segment_ids = features[\"segment_ids\"]\n",
    "        label_ids = features[\"label_ids\"]\n",
    "\n",
    "        is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "\n",
    "        (total_loss, per_example_loss, logits, probabilities) = create_model(\n",
    "            bert_config, is_training, input_ids, input_mask, segment_ids, label_ids,\n",
    "            num_labels, use_one_hot_embeddings)\n",
    "\n",
    "        tvars = tf.trainable_variables()\n",
    "        initialized_variable_names = {}\n",
    "        scaffold_fn = None\n",
    "        if init_checkpoint:\n",
    "            (assignment_map, initialized_variable_names\n",
    "             ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)\n",
    "            if use_tpu:\n",
    "\n",
    "                def tpu_scaffold():\n",
    "                    tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n",
    "                    return tf.train.Scaffold()\n",
    "\n",
    "                scaffold_fn = tpu_scaffold\n",
    "            else:\n",
    "                tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n",
    "\n",
    "        tf.logging.info(\"**** Trainable Variables ****\")\n",
    "        for var in tvars:\n",
    "            init_string = \"\"\n",
    "            if var.name in initialized_variable_names:\n",
    "                init_string = \", *INIT_FROM_CKPT*\"\n",
    "            tf.logging.info(\"  name = %s, shape = %s%s\", var.name, var.shape,\n",
    "                            init_string)\n",
    "\n",
    "        output_spec = None\n",
    "        if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "            predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)\n",
    "\n",
    "            accuracy = tf.metrics.accuracy(label_ids, predictions)\n",
    "            tf.summary.scalar('accuracy', accuracy[1])\n",
    "\n",
    "            train_op = optimization.create_optimizer(\n",
    "                total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)\n",
    "\n",
    "            output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
    "                mode=mode,\n",
    "                loss=total_loss,\n",
    "                train_op=train_op,\n",
    "                scaffold_fn=scaffold_fn)\n",
    "\n",
    "        elif mode == tf.estimator.ModeKeys.EVAL:\n",
    "\n",
    "            def metric_fn(per_example_loss, label_ids, logits):\n",
    "                predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)\n",
    "\n",
    "                accuracy = tf.metrics.accuracy(label_ids, predictions)\n",
    "                loss = tf.metrics.mean(per_example_loss)\n",
    "                return {\n",
    "                    \"eval_accuracy\": accuracy,\n",
    "                    \"eval_loss\": loss,\n",
    "                }\n",
    "\n",
    "            eval_metrics = (metric_fn, [per_example_loss, label_ids, logits])\n",
    "            output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
    "                mode=mode,\n",
    "                loss=total_loss,\n",
    "                eval_metrics=eval_metrics,\n",
    "                scaffold_fn=scaffold_fn)\n",
    "        else:\n",
    "            output_spec = tf.contrib.tpu.TPUEstimatorSpec(\n",
    "                mode=mode, predictions=probabilities, scaffold_fn=scaffold_fn)\n",
    "        return output_spec\n",
    "\n",
    "    return model_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def file_based_input_fn_builder(input_file, seq_length, is_training,\n",
    "                                drop_remainder):\n",
    "    \"\"\"Creates an `input_fn` closure to be passed to TPUEstimator.\"\"\"\n",
    "\n",
    "    name_to_features = {\n",
    "        \"input_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        \"input_mask\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        \"segment_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n",
    "        \"label_ids\": tf.FixedLenFeature([], tf.int64),\n",
    "    }\n",
    "\n",
    "    def _decode_record(record, name_to_features):\n",
    "        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n",
    "        example = tf.parse_single_example(record, name_to_features)\n",
    "\n",
    "        # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n",
    "        # So cast all int64 to int32.\n",
    "        for name in list(example.keys()):\n",
    "            t = example[name]\n",
    "            if t.dtype == tf.int64:\n",
    "                t = tf.to_int32(t)\n",
    "            example[name] = t\n",
    "\n",
    "        return example\n",
    "\n",
    "    def input_fn(params):\n",
    "        \"\"\"The actual input function.\"\"\"\n",
    "        batch_size = params[\"batch_size\"]\n",
    "\n",
    "        # For training, we want a lot of parallel reading and shuffling.\n",
    "        # For eval, we want no shuffling and parallel reading doesn't matter.\n",
    "        d = tf.data.TFRecordDataset(input_file)\n",
    "        if is_training:\n",
    "            d = d.repeat()\n",
    "            d = d.shuffle(buffer_size=100)\n",
    "\n",
    "        d = d.apply(\n",
    "            tf.contrib.data.map_and_batch(\n",
    "                lambda record: _decode_record(record, name_to_features),\n",
    "                batch_size=batch_size,\n",
    "                drop_remainder=drop_remainder))\n",
    "\n",
    "        return d\n",
    "\n",
    "    return input_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bert_config = modeling.BertConfig.from_json_file(bert_config_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<modeling.BertConfig at 0x7fe52b618f60>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_examples, truth_labels, placeholder_labels = create_examples(rc.DataProcessor._read_tsv(train_dir),\"train\")\n",
    "predict_file = os.path.join(output_dir, \"predict.tf_record\")\n",
    "\n",
    "label_list_test = set(placeholder_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_fn=rc.model_fn_builder(\n",
    "        bert_config=bert_config,\n",
    "        # value is 2\n",
    "        num_labels=len(label_list_test),\n",
    "        init_checkpoint=init_checkpoint,\n",
    "        learning_rate=learning_rate,\n",
    "        num_train_steps=num_train_steps,\n",
    "        num_warmup_steps=num_warmup_steps,\n",
    "        use_tpu=use_tpu,\n",
    "        use_one_hot_embeddings=use_tpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "run_config = tf.contrib.tpu.RunConfig(\n",
    "    cluster=tpu_cluster_resolver,\n",
    "    master=master,\n",
    "    model_dir=output_dir,\n",
    "    save_checkpoints_steps=save_checkpoints_steps,\n",
    "    tpu_config=tf.contrib.tpu.TPUConfig(\n",
    "        iterations_per_loop=iterations_per_loop,\n",
    "        num_shards=num_tpu_cores,\n",
    "        per_host_input_for_training=is_per_host))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x7fe52b9f5b70>) includes params argument, but params are not passed to Estimator.\n",
      "INFO:tensorflow:Using config: {'_model_dir': './output_models/', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': 50, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fe529f98f60>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=50, num_shards=8, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n",
      "INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
      "WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n"
     ]
    }
   ],
   "source": [
    "estimator = tf.contrib.tpu.TPUEstimator(\n",
    "    use_tpu=use_tpu,\n",
    "    model_fn=model_fn,\n",
    "    config=run_config,\n",
    "    train_batch_size=train_batch_size,\n",
    "    eval_batch_size=eval_batch_size,\n",
    "    predict_batch_size=predict_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Running prediction*****\n",
      "INFO:tensorflow:  Num examples = 39630\n",
      "INFO:tensorflow:  Batch size = 8\n"
     ]
    }
   ],
   "source": [
    "tf.logging.info(\"***** Running prediction*****\")\n",
    "tf.logging.info(\"  Num examples = %d\", len(predict_examples))\n",
    "tf.logging.info(\"  Batch size = %d\", predict_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_input_fn = file_based_input_fn_builder(\n",
    "    input_file=predict_file,\n",
    "    seq_length=max_seq_length,\n",
    "    is_training=False,\n",
    "    drop_remainder=predict_drop_remainder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result = estimator.predict(input_fn=predict_input_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:***** Predict results *****\n",
      "WARNING:tensorflow:From <ipython-input-8-5da4aab07e5c>:41: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.data.experimental.map_and_batch(...)`.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Running infer on CPU\n",
      "INFO:tensorflow:*** Features ***\n",
      "INFO:tensorflow:  name = input_ids, shape = (?, 128)\n",
      "INFO:tensorflow:  name = input_mask, shape = (?, 128)\n",
      "INFO:tensorflow:  name = label_ids, shape = (?,)\n",
      "INFO:tensorflow:  name = segment_ids, shape = (?, 128)\n",
      "INFO:tensorflow:**** Trainable Variables ****\n",
      "INFO:tensorflow:  name = bert/embeddings/word_embeddings:0, shape = (30522, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/token_type_embeddings:0, shape = (2, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/position_embeddings:0, shape = (512, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/embeddings/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_0/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_1/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_2/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_3/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_4/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_5/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_6/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_7/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_8/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_9/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_10/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/query/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/key/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/self/value/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/attention/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/kernel:0, shape = (768, 3072), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/intermediate/dense/bias:0, shape = (3072,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/kernel:0, shape = (3072, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/beta:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/encoder/layer_11/output/LayerNorm/gamma:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/pooler/dense/kernel:0, shape = (768, 768), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = bert/pooler/dense/bias:0, shape = (768,), *INIT_FROM_CKPT*\n",
      "INFO:tensorflow:  name = output_weights:0, shape = (7, 768)\n",
      "INFO:tensorflow:  name = output_bias:0, shape = (7,)\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from ./output_models/model.ckpt-3715\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:prediction_loop marked as finished\n",
      "INFO:tensorflow:prediction_loop marked as finished\n"
     ]
    }
   ],
   "source": [
    "output_predict_file = os.path.join(output_dir, \"test_results_hehe.tsv\")\n",
    "with tf.gfile.GFile(output_predict_file, \"w\") as writer:\n",
    "    tf.logging.info(\"***** Predict results *****\")\n",
    "    for prediction in result:\n",
    "        output_line = \"\\t\".join(\n",
    "            str(class_probability) for class_probability in prediction) + \"\\n\"\n",
    "        writer.write(output_line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./output_models/test_results_hehe.tsv'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_predict_file "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Theres big V hood Lincoln MKS secret turbochar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The optional Technology package uses adaptive ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Active headrests also included The federal gov...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Major options include inch steel wheels power ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Electrons alone power Fiat e roughly miles acc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>All Mazda CX powered liter V produces horsepow...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>EPA rated fuel economy Compass mpg city mpg hi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>EPA estimated fuel economy Series depends engi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>It common expensive sports cars miss official ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Passing power much better Honda VTEC technolog...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Merging passing maneuvers require little effor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>The reviewer remarks surprisingly responsive r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Antilock disc brakes standard stability contro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>EPA estimated fuel economy mpg city mpg highwa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>PerformanceThe Range Rover Sport powerful engi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>The Insurance Institute Highway Safety gave be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>A small screen behind steering wheel sits prou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>The A Premium comes standard inch alloy wheels...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Every BMW X comes standard antilock disc brake...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>The Ford Expedition full size body frame SUV a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>The Competition package sharpens M responses b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>The AWD T rate miles per gallon less</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>The GT includes amenities adds active front di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Only one engine offered XC wagon liter turboch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>That technology think limited application safety</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>The Freedom Drive II Off Road Group available ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>The pricey LTZ trim dresses cabin significantl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>The F comes standard antilock brakes traction ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Standard features include inch alloy wheels ai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Door reinforcements bolster coupes safety side...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30356</th>\n",
       "      <td>Every Dodge Challenger comes standard antilock...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30357</th>\n",
       "      <td>The GLS interior storage offerings dont stand ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30358</th>\n",
       "      <td>The fuel economy estimates come mpg city mpg h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30359</th>\n",
       "      <td>The Premium includes equipment adds panoramic ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30360</th>\n",
       "      <td>The look blend tech formal elegance digital ga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30361</th>\n",
       "      <td>Ergonomics great switches functions like hazar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30362</th>\n",
       "      <td>Fuel economy rear drive four cylinder Tacoma a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30363</th>\n",
       "      <td>Highlights include leather seating park assist...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30364</th>\n",
       "      <td>You problem finding better gas mileage four cy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30365</th>\n",
       "      <td>Front side side curtain airbags standard ABS t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30366</th>\n",
       "      <td>Standard electronics features include iDrive e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30367</th>\n",
       "      <td>The Highlander large three row crossover SUV s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30368</th>\n",
       "      <td>Ford EcoBoost V example churns hp turbocharged...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30369</th>\n",
       "      <td>The hatchback body style provides easy access ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30370</th>\n",
       "      <td>EPA fuel economy estimates manual transmission...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30371</th>\n",
       "      <td>The front wheel drive VW Eos powered liter tur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30372</th>\n",
       "      <td>The XLT adds upgraded brakes foglights rear pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30373</th>\n",
       "      <td>During Edmunds performance testing Lancer SEL ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30374</th>\n",
       "      <td>Its sharp creases pinched body lines unique NX...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30375</th>\n",
       "      <td>The Kia Soul remains surprisingly good safety ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30376</th>\n",
       "      <td>The inline engine turbocharged direct injected...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30377</th>\n",
       "      <td>They opt contrast stitching add subtract brogu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30378</th>\n",
       "      <td>The V pulls hard sounds amazing dual clutch tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30379</th>\n",
       "      <td>With introduction four cylinder model actually...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30380</th>\n",
       "      <td>However Road amp Track says Speed model also s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30381</th>\n",
       "      <td>What think Hybrid even feels better balanced t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30382</th>\n",
       "      <td>A three position cargo cover separates passeng...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30383</th>\n",
       "      <td>To get kind economy move size class Sonata Hyb...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30384</th>\n",
       "      <td>It also foglights keyless entry removable full...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30385</th>\n",
       "      <td>The SE Titanium come standard turbocharged lit...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30386 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    text\n",
       "0      Theres big V hood Lincoln MKS secret turbochar...\n",
       "1      The optional Technology package uses adaptive ...\n",
       "2      Active headrests also included The federal gov...\n",
       "3      Major options include inch steel wheels power ...\n",
       "4      Electrons alone power Fiat e roughly miles acc...\n",
       "5      All Mazda CX powered liter V produces horsepow...\n",
       "6      EPA rated fuel economy Compass mpg city mpg hi...\n",
       "7      EPA estimated fuel economy Series depends engi...\n",
       "8      It common expensive sports cars miss official ...\n",
       "9      Passing power much better Honda VTEC technolog...\n",
       "10     Merging passing maneuvers require little effor...\n",
       "11     The reviewer remarks surprisingly responsive r...\n",
       "12     Antilock disc brakes standard stability contro...\n",
       "13     EPA estimated fuel economy mpg city mpg highwa...\n",
       "14     PerformanceThe Range Rover Sport powerful engi...\n",
       "15     The Insurance Institute Highway Safety gave be...\n",
       "16     A small screen behind steering wheel sits prou...\n",
       "17     The A Premium comes standard inch alloy wheels...\n",
       "18     Every BMW X comes standard antilock disc brake...\n",
       "19     The Ford Expedition full size body frame SUV a...\n",
       "20     The Competition package sharpens M responses b...\n",
       "21                  The AWD T rate miles per gallon less\n",
       "22     The GT includes amenities adds active front di...\n",
       "23     Only one engine offered XC wagon liter turboch...\n",
       "24      That technology think limited application safety\n",
       "25     The Freedom Drive II Off Road Group available ...\n",
       "26     The pricey LTZ trim dresses cabin significantl...\n",
       "27     The F comes standard antilock brakes traction ...\n",
       "28     Standard features include inch alloy wheels ai...\n",
       "29     Door reinforcements bolster coupes safety side...\n",
       "...                                                  ...\n",
       "30356  Every Dodge Challenger comes standard antilock...\n",
       "30357  The GLS interior storage offerings dont stand ...\n",
       "30358  The fuel economy estimates come mpg city mpg h...\n",
       "30359  The Premium includes equipment adds panoramic ...\n",
       "30360  The look blend tech formal elegance digital ga...\n",
       "30361  Ergonomics great switches functions like hazar...\n",
       "30362  Fuel economy rear drive four cylinder Tacoma a...\n",
       "30363  Highlights include leather seating park assist...\n",
       "30364  You problem finding better gas mileage four cy...\n",
       "30365  Front side side curtain airbags standard ABS t...\n",
       "30366  Standard electronics features include iDrive e...\n",
       "30367  The Highlander large three row crossover SUV s...\n",
       "30368  Ford EcoBoost V example churns hp turbocharged...\n",
       "30369  The hatchback body style provides easy access ...\n",
       "30370  EPA fuel economy estimates manual transmission...\n",
       "30371  The front wheel drive VW Eos powered liter tur...\n",
       "30372  The XLT adds upgraded brakes foglights rear pa...\n",
       "30373  During Edmunds performance testing Lancer SEL ...\n",
       "30374  Its sharp creases pinched body lines unique NX...\n",
       "30375  The Kia Soul remains surprisingly good safety ...\n",
       "30376  The inline engine turbocharged direct injected...\n",
       "30377  They opt contrast stitching add subtract brogu...\n",
       "30378  The V pulls hard sounds amazing dual clutch tr...\n",
       "30379  With introduction four cylinder model actually...\n",
       "30380  However Road amp Track says Speed model also s...\n",
       "30381  What think Hybrid even feels better balanced t...\n",
       "30382  A three position cargo cover separates passeng...\n",
       "30383  To get kind economy move size class Sonata Hyb...\n",
       "30384  It also foglights keyless entry removable full...\n",
       "30385  The SE Titanium come standard turbocharged lit...\n",
       "\n",
       "[30386 rows x 1 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(data_dir,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
